{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11598, 471)\n"
     ]
    }
   ],
   "source": [
    "# 加载数据\n",
    "import numpy as np \n",
    "# import sklean \n",
    "\n",
    "dataset = np.loadtxt('../data/maldroid2020/feature_vectors_syscallsbinders_frequency_5_Cat.csv',dtype=float,skiprows=1,delimiter=',')\n",
    "\n",
    "print(dataset.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3    2677\n",
      "4    1868\n",
      "2    1459\n",
      "5    1257\n",
      "1     857\n",
      "dtype: int64\n",
      "3    1227\n",
      "4     678\n",
      "2     641\n",
      "5     538\n",
      "1     396\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "import pandas as pd \n",
    "# 对数据进行处理和分割\n",
    "X,y = dataset[:,:-1],dataset[:,-1].astype(int)\n",
    "# print(X.shape,y.shape)\n",
    "X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.3)\n",
    "print(pd.value_counts(y_train))\n",
    "print(pd.value_counts(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8307471264367816\n"
     ]
    }
   ],
   "source": [
    "# 1使用KNN进行分类\n",
    "from sklearn import neighbors\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "knn_classifier = neighbors.KNeighborsClassifier(n_neighbors=5,weights='uniform')\n",
    "\n",
    "knn_classifier.fit(X_train,y_train)\n",
    "y_pred = knn_classifier.predict(X_test)\n",
    "knn_score = accuracy_score(y_test,y_pred,normalize=True)\n",
    "print(knn_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9011494252873563 15\n"
     ]
    }
   ],
   "source": [
    "# 2使用决策树进行分类\n",
    "from sklearn import tree \n",
    "from sklearn.metrics import accuracy_score\n",
    "dt_classifier = tree.DecisionTreeClassifier(max_depth=15,criterion='entropy')\n",
    "dt_classifier.fit(X=X_train ,y=y_train)\n",
    "y_pred = dt_classifier.predict(X_test)\n",
    "dt_score = accuracy_score(y_test,y_pred)\n",
    "print(dt_score,dt_classifier.get_depth())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7451149425287357\n"
     ]
    }
   ],
   "source": [
    "# 3使用bayes进行分类\n",
    "from sklearn import naive_bayes\n",
    "# nb_classifier = naive_bayes.GaussianNB()\n",
    "# nb_classifier = naive_bayes.MultinomialNB()\n",
    "nb_classifier = naive_bayes.BernoulliNB()\n",
    "nb_classifier.fit(X_train,y_train)\n",
    "y_pred = nb_classifier.predict(X_test)\n",
    "nb_score = accuracy_score(y_test,y_pred)\n",
    "print(nb_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5482758620689655\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ykl/.virtualenvs/pysyft/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "source": [
    "# 4使用逻辑回归.本身是不能多分类的。所以准确率很低。\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "lr_classifier = LogisticRegression()\n",
    "lr_classifier.fit(X_train,y_train)\n",
    "y_pred = lr_classifier.predict(X_test)\n",
    "lr_score = accuracy_score(y_test,y_pred)\n",
    "print(lr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5使用svm-linear\n",
    "# from sklearn import svm\n",
    "# svm_classifier = svm.SVC(kernel='linear')\n",
    "# svm_classifier.fit(X_train,y_train)\n",
    "# y_pred = svm_classifier.predict(X_test)\n",
    "# svm_score = accuracy_score(y_test,y_pred)\n",
    "# print(svm_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用svm-rbf\n",
    "# from sklearn import svm\n",
    "# svm_classifier = svm.SVC(kernel='poly',degree=5)\n",
    "# svm_classifier.fit(X_train,y_train)\n",
    "# y_pred = svm_classifier.predict(X_test)\n",
    "# accuracy_score(y_test,y_pred)\n",
    "# print(svm_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9468390804597702\n"
     ]
    }
   ],
   "source": [
    "# 6.1使用随机森林进行分类\n",
    "from sklearn import ensemble\n",
    "rf_classifier = ensemble.RandomForestClassifier()\n",
    "rf_classifier.fit(X_train,y_train)\n",
    "y_pred = rf_classifier.predict(X_test)\n",
    "rf_score = accuracy_score(y_test,y_pred)\n",
    "print(rf_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.907183908045977\n"
     ]
    }
   ],
   "source": [
    "# 6.2使用adaboost\n",
    "from sklearn import ensemble\n",
    "from sklearn import tree \n",
    "dt = tree.DecisionTreeClassifier(max_depth=5,criterion='entropy')\n",
    "ab_classifier = ensemble.AdaBoostClassifier(dt)\n",
    "ab_classifier.fit(X_train,y_train)\n",
    "y_pred = ab_classifier.predict(X_test)\n",
    "ab_score = accuracy_score(y_test,y_pred)\n",
    "print(ab_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.421551724137931\n"
     ]
    }
   ],
   "source": [
    "# 7使用SGD+SVM\n",
    "from sklearn import linear_model\n",
    "sgd_classifier = linear_model.SGDClassifier(loss='squared_hinge',l1_ratio=0.15,learning_rate='optimal',n_jobs=5,penalty='l2',random_state=True)\n",
    "sgd_classifier.fit(X_train,y_train)\n",
    "y_pred = sgd_classifier.predict(X_test)\n",
    "sgd_score = accuracy_score(y_test,y_pred)\n",
    "print(sgd_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9031609195402299\n"
     ]
    }
   ],
   "source": [
    "# 8使用OneVsRestClassifier+logisticsRegression进行多分类\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "ovr_classifier = OneVsRestClassifier(DecisionTreeClassifier(max_depth=10,criterion='entropy'),n_jobs=5)\n",
    "ovr_classifier.fit(X_train,y_train)\n",
    "y_pred = ovr_classifier.predict(X_test)\n",
    "ovr_score = accuracy_score(y_test,y_pred)\n",
    "print(ovr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2.79117602e-01 4.18133909e-03 1.01426846e-02 3.81305173e-01\n",
      " 1.58147295e-01 3.74638249e-01 3.75907048e-01 1.79174048e-01\n",
      " 4.77030852e-01 2.55402938e-01 3.67190031e-01 2.43662207e-02\n",
      " 1.10865182e-01 1.35728382e-01 3.10874557e-01 7.17263939e-02\n",
      " 4.53025532e-01 7.68244861e-03 2.38074416e-01 2.73925480e-01\n",
      " 3.65416134e-03 1.13981422e-01 8.65741957e-02 1.89354443e-01\n",
      " 1.19933163e-01 2.18043291e-01 2.14286354e-01 5.22160041e-04\n",
      " 2.37119184e-01 0.00000000e+00 1.48562739e-01 7.49955478e-02\n",
      " 2.61423761e-02 1.52392344e-01 3.09460320e-01 1.46062880e-01\n",
      " 3.84614408e-01 3.73549421e-01 2.36731011e-01 2.29892683e-03\n",
      " 5.78980346e-01 1.23640206e-02 9.48214930e-02 3.07604504e-03\n",
      " 7.37515850e-02 1.30326386e-02 0.00000000e+00 1.25706658e-01\n",
      " 3.55964279e-02 0.00000000e+00 0.00000000e+00 1.38298504e-03\n",
      " 2.33351602e-01 3.55952775e-01 2.84895945e-03 2.55972491e-02\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 6.54943210e-03\n",
      " 3.17605273e-02 8.21896099e-02 4.71135467e-02 1.40644026e-01\n",
      " 8.33516249e-03 3.18842667e-01 1.68091564e-03 4.17685193e-03\n",
      " 4.35526873e-01 3.73483960e-01 5.51934797e-01 3.51756658e-03\n",
      " 2.31520382e-01 0.00000000e+00 0.00000000e+00 1.09162830e-02\n",
      " 5.95378326e-03 5.19899035e-03 3.06567770e-01 1.14929551e-01\n",
      " 0.00000000e+00 0.00000000e+00 1.26314285e-02 3.34243818e-02\n",
      " 2.10076385e-01 2.92710189e-01 4.68178350e-01 0.00000000e+00\n",
      " 1.67937531e-01 3.17999331e-01 1.58334446e-01 0.00000000e+00\n",
      " 2.40134288e-01 2.65824495e-01 0.00000000e+00 5.57349699e-01\n",
      " 2.73301561e-01 1.79838784e-01 2.05380117e-03 3.50183874e-03\n",
      " 1.95716127e-01 1.42680345e-01 5.20923923e-01 0.00000000e+00\n",
      " 2.74271864e-01 8.06454913e-02 9.66437447e-02 3.20126263e-01\n",
      " 0.00000000e+00 9.55142221e-03 1.02165299e-02 0.00000000e+00\n",
      " 2.32145475e-01 2.20528632e-01 1.43768467e-01 1.74550866e-03\n",
      " 7.58676462e-02 0.00000000e+00 1.07569009e-02 4.69243047e-02\n",
      " 1.20361549e-02 0.00000000e+00 4.47435243e-04 3.23052121e-01\n",
      " 5.17486530e-02 5.72435577e-03 1.19583784e-02 7.24444216e-03\n",
      " 4.10089657e-03 0.00000000e+00 6.40301888e-03 1.17089180e-01\n",
      " 6.72194754e-03 3.64222829e-03 0.00000000e+00 1.41508188e-01\n",
      " 0.00000000e+00 0.00000000e+00 7.68682576e-03 2.52245482e-02\n",
      " 0.00000000e+00 1.00585605e-01 1.61202854e-02 2.02623448e-01\n",
      " 1.84679497e-03 1.05568467e-02 1.37243560e-02 0.00000000e+00\n",
      " 1.36297753e-02 1.41514738e-02 3.54176242e-01 4.11789252e-03\n",
      " 6.66335768e-03 9.18012641e-03 7.91003499e-03 6.02734521e-03\n",
      " 9.60285190e-03 0.00000000e+00 9.72126457e-03 1.31214997e-01\n",
      " 1.40774493e-01 4.01298583e-02 3.27709591e-02 1.41838307e-02\n",
      " 0.00000000e+00 4.75914821e-02 1.18657558e-01 0.00000000e+00\n",
      " 1.77715709e-01 0.00000000e+00 5.67316790e-03 3.96244156e-03\n",
      " 3.84179948e-02 1.62623481e-01 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 7.33843330e-03 0.00000000e+00 2.79031900e-02\n",
      " 0.00000000e+00 1.79384469e-02 1.01455363e-01 2.91323679e-03\n",
      " 2.90977734e-02 1.05624412e-02 0.00000000e+00 3.27219631e-01\n",
      " 1.54806086e-03 7.38978996e-03 7.75066139e-03 0.00000000e+00\n",
      " 0.00000000e+00 5.48864264e-04 0.00000000e+00 3.57901302e-03\n",
      " 5.75166569e-04 0.00000000e+00 1.47476396e-02 0.00000000e+00\n",
      " 1.21195915e-01 2.66273074e-02 1.22329942e-04 1.05762171e-02\n",
      " 8.39947080e-03 6.77538179e-03 1.18730603e-01 1.13467512e-02\n",
      " 1.67278027e-02 1.03656281e-02 1.79602833e-02 4.92597079e-02\n",
      " 0.00000000e+00 1.93718978e-01 3.41763354e-03 1.58743717e-02\n",
      " 7.14112111e-03 0.00000000e+00 0.00000000e+00 8.59800256e-03\n",
      " 2.21947796e-03 0.00000000e+00 8.49111941e-03 0.00000000e+00\n",
      " 7.60520827e-03 2.17468021e-03 0.00000000e+00 2.87740341e-03\n",
      " 2.69909292e-03 0.00000000e+00 4.97285599e-03 0.00000000e+00\n",
      " 3.98042808e-02 9.43901232e-02 7.03960616e-03 3.04954595e-01\n",
      " 7.46666098e-02 7.92991355e-02 7.86502575e-02 0.00000000e+00\n",
      " 9.79078749e-03 1.03793633e-02 4.95547807e-01 8.38529002e-03\n",
      " 3.90351836e-01 4.16957532e-03 5.29740581e-03 1.00085107e-02\n",
      " 2.72695179e-01 3.03842331e-01 5.17839158e-01 6.32310639e-01\n",
      " 4.90200514e-01 3.94998788e-03 2.96180264e-03 1.28458315e-02\n",
      " 0.00000000e+00 5.74364340e-02 9.75902821e-03 1.32334495e-01\n",
      " 0.00000000e+00 1.60867155e-02 2.66251189e-03 4.36887569e-03\n",
      " 0.00000000e+00 0.00000000e+00 3.69130602e-01 3.15778012e-02\n",
      " 1.43007838e-01 4.23049435e-04 0.00000000e+00 1.09510607e-03\n",
      " 3.36751735e-02 4.82374433e-04 4.94549663e-04 7.23517751e-04\n",
      " 1.43866985e-02 3.30033412e-03 7.94712264e-03 1.41362964e-01\n",
      " 8.06508212e-02 2.93506806e-02 1.08042412e-02 1.35115725e-02\n",
      " 3.27282523e-03 2.04878661e-03 4.14713682e-02 0.00000000e+00\n",
      " 4.59864074e-01 4.19167327e-01 4.20570040e-01 3.73470755e-01\n",
      " 6.08493080e-03 1.32950010e-04 5.95471639e-01 0.00000000e+00\n",
      " 4.78554709e-01 5.71674931e-03 1.17530330e-01 0.00000000e+00\n",
      " 5.70836000e-01 2.46659248e-01 0.00000000e+00 2.39640060e-02\n",
      " 0.00000000e+00 0.00000000e+00 5.83659012e-03 6.71928025e-03\n",
      " 4.28518542e-02 4.95261836e-03 1.10785779e-02 5.82731806e-01\n",
      " 4.33585434e-02 8.12177965e-03 3.49024093e-02 2.23526566e-03\n",
      " 2.73727352e-01 1.24697712e-02 1.30371254e-04 0.00000000e+00\n",
      " 0.00000000e+00 2.22636539e-01 3.89450195e-01 6.16867958e-01\n",
      " 0.00000000e+00 0.00000000e+00 3.97005713e-01 5.98735687e-03\n",
      " 6.23085057e-02 0.00000000e+00 3.35183919e-02 9.71979074e-02\n",
      " 5.33311405e-01 6.39226775e-03 3.83773874e-03 4.15278249e-01\n",
      " 2.15503324e-01 4.79491730e-03 1.17085388e-02 1.38757543e-01\n",
      " 3.56591420e-02 0.00000000e+00 9.03482681e-03 2.69808675e-03\n",
      " 2.00481521e-01 0.00000000e+00 2.41076888e-02 2.56759803e-03\n",
      " 1.21864954e-01 1.90887636e-03 2.69488668e-03 0.00000000e+00\n",
      " 2.90160472e-03 0.00000000e+00 2.71820686e-01 1.70642274e-02\n",
      " 0.00000000e+00 6.82759666e-02 4.29819121e-02 1.92428078e-02\n",
      " 0.00000000e+00 1.34388406e-02 0.00000000e+00 3.03866274e-04\n",
      " 0.00000000e+00 0.00000000e+00 3.51195123e-02 3.81921696e-02\n",
      " 0.00000000e+00 0.00000000e+00 1.70076041e-01 9.16383344e-02\n",
      " 9.34579788e-02 2.87209680e-03 0.00000000e+00 1.93790007e-02\n",
      " 2.13777158e-01 2.63260565e-01 6.38485014e-02 0.00000000e+00\n",
      " 9.61855042e-03 9.35622596e-02 5.24495245e-03 8.68271362e-03\n",
      " 0.00000000e+00 1.49088513e-03 1.91414014e-02 5.69757936e-02\n",
      " 0.00000000e+00 0.00000000e+00 3.16143478e-02 5.79977314e-02\n",
      " 0.00000000e+00 1.14993071e-02 0.00000000e+00 0.00000000e+00\n",
      " 1.04144293e-02 4.70680216e-03 2.06049476e-02 4.84302684e-03\n",
      " 5.51566991e-03 0.00000000e+00 0.00000000e+00 1.78114548e-02\n",
      " 2.49202664e-02 1.69594787e-02 0.00000000e+00 9.88154449e-03\n",
      " 1.09504192e-02 2.58593193e-03 6.73071806e-03 3.21436409e-02\n",
      " 4.29078492e-03 8.04897174e-03 4.26691910e-03 1.22923069e-01\n",
      " 3.89686002e-01 1.40952775e-02 4.55729066e-03 6.07539644e-03\n",
      " 1.85078518e-03 3.05916689e-01 5.46998419e-03 1.11595550e-02\n",
      " 0.00000000e+00 9.44772994e-02 1.82192385e-01 4.17994852e-01\n",
      " 5.41289961e-01 6.45703079e-03 2.65476763e-01 1.13521109e-01\n",
      " 1.08085609e-02 3.63035545e-02 5.12698353e-03 2.52104782e-03\n",
      " 8.91821000e-03 5.49407427e-01 2.84866304e-01 1.00875255e-02\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 1.06263578e-02 2.46144137e-02 0.00000000e+00 5.55635451e-03\n",
      " 2.31281301e-03 2.08300883e-03 1.54212024e-01 1.10742754e-02\n",
      " 4.57004749e-02 2.88734330e-01 2.40858957e-02 0.00000000e+00\n",
      " 0.00000000e+00 2.23367993e-03 3.14950204e-03 2.74508221e-03\n",
      " 0.00000000e+00 5.01378141e-03 4.88346848e-02 0.00000000e+00\n",
      " 4.29266237e-03 1.69204226e-01 1.82866430e-03 1.36484450e-01\n",
      " 4.20718117e-01 4.01839225e-01]\n"
     ]
    }
   ],
   "source": [
    "# 对数据内容进行统计——信息增益\n",
    "from sklearn.feature_selection import mutual_info_classif\n",
    "mutual_increase = mutual_info_classif(X,y)\n",
    "print(mutual_increase)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8 [0.38063927 0.29637107 0.1408946  0.05674946 0.04825538 0.03905179\n",
      " 0.01014331 0.00895998]\n"
     ]
    }
   ],
   "source": [
    "# PCA降维\n",
    "from sklearn import decomposition\n",
    "pca = decomposition.PCA(n_components=0.98)\n",
    "\n",
    "data_pca = pca.fit_transform(X)\n",
    "print(pca.n_components_,pca.explained_variance_ratio_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8497126436781609\n"
     ]
    }
   ],
   "source": [
    "# 对降维后的数据使用随机森林看一下。对准确率的影响没有想象中那么大。\n",
    "from sklearn import ensemble\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "x1,x2,y1,y2 = train_test_split(data_pca,y,test_size=0.3)\n",
    "rf_classifier2 = ensemble.RandomForestClassifier()\n",
    "rf_classifier2.fit(x1,y1)\n",
    "y_pred = rf_classifier2.predict(x2)\n",
    "rf_score2 = accuracy_score(y2,y_pred)\n",
    "print(rf_score2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.9 64-bit ('pysyft': virtualenvwrapper)",
   "name": "python3810jvsc74a57bd054f0cca124f38538f0af9da5035feb8330e039430efda7f6532c7bec2bbfbd8a"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 2
 },
 "nbformat": 4,
 "nbformat_minor": 2
}